package com.g2room.driving.grap.life;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.grap.life.model.po.LifeShop;
import com.g2room.driving.util.DateUtil;
import com.g2room.driving.util.ImageUtil;
import com.g2room.myuall.grap.ky.model.po.GrapErr;
import com.g2room.myuall.grap.univ.po.LifeNews;
import com.g2room.myuall.util.DbUtil;
import com.g2room.myuall.util.HtmlUtil;

public class GrapWangshangcunzhuangData {
	private Connection conn = null;
	public String baseUrl = "http://www.17wscz.com";
	private String source = "网上村庄";
//	public String baseUrl = "http://www.cqnjl.com";
//	private String source = "城乡游";
//	private static AtomicInteger id = new AtomicInteger(getMaxId());
	private static Map<String, Boolean> titles = new HashMap<String, Boolean>();
	public static void main(String[] args) throws Exception{
		GrapWangshangcunzhuangData gkd = new GrapWangshangcunzhuangData();
//		gkd.getNjlPage("http://www.17wscz.com/Farm/List?VendorType=2&isLoadAttributeModels=true&isLoadTag=true&pageIndex=#page#", 106);
		gkd.getCqNjlPage("http://www.cqnjl.cn/nongjiale-6-#page#-----1", 62);
    }

	private void getCqNjlPage(String url, int page) {
		//#page#
		for(int i =1; i< page+1; i++) {
			getCqNjlList(url.replace("#page#", i+""));
		}
	}
	private void getCqNjlList(String url) {
		Document doc = null;
		try {
			doc = GenericGrap.getPostDocByIp(url);
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("1");
			ge.setDescription("获取List 错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		List<LifeShop> lns = new ArrayList<LifeShop>();
		Elements tels = doc.select(".lixtqy li a");
		if(null != tels && !tels.isEmpty()) {
			for(Element el : tels) {
				LifeShop ls = new LifeShop();
				// 获取封面
				Elements els = el.select("img");
				if(null != els && !els.isEmpty()) {
					ls.setCover(els.first().attr("src"));
				} else {
					continue;
				}
				/**
				 <a href="http://www.cqnjl.cn/nongjiale_6_371" title="醉美东山生态农庄" target="_blank">
            <img src="http://www.cqnjl.cn/uploadfile/2014/0421/thumb_227_153_20140421015600229.jpg" title="醉美东山生态农庄" alt="醉美东山生态农庄" width="227" height="153">
        </a>
				 */
				ls.setSource("城乡游");
				ls.setSourceUrl(el.attr("href"));
				ls.setTypeName("农家乐");
				
				lns.add(ls);
			}
		}
		
		for(LifeShop li : lns) {
			try {
				getCqNjlDetail(li);
			} catch (Exception e) {
				lns.remove(li);
				e.printStackTrace();
			}
		}
		
		if(lns.size() > 0) insert2ShopDb(lns);
	}

	private LifeShop getCqNjlDetail(LifeShop ls) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocumentByIp(ls.getSourceUrl());
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(ls.getSourceUrl());
			ge.setType("5");
			ge.setContent(JSON.toJSONString(ls));
			ge.setDescription("获取Detail 错误："+e.getMessage());
			insert2ErrDb(ge);
			return null;
		}
		Elements els = doc.select(".zhuti_right p em");
		if(null != els && !els.isEmpty()) {
			ls.setName(els.first().text());
		}
		
		els = doc.select(".zhuti_right p.sjjs_mb");
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				Elements tels = el.select("span");
				if(null != tels && !tels.isEmpty()) {
					Element tel = tels.first();
					if(tel.text().contains("服务特色")) {
						tel.remove();
						String tags = "";
						for(Element ttel : tels) {
							if(ttel.text().contains("服务特色")) continue;
							if(tags.length() > 0) tags += ";";
							tags+=ttel.text().trim();
						}
						ls.setFlavor(tags);
					} else if(tel.text().contains("适合场合")) {
						tel.remove();
						String tags = "";
						for(Element ttel : tels) {
							if(ttel.text().contains("适合场合")) continue;
							if(tags.length() > 0) tags += ";";
							tags+=ttel.text().trim();
						}
						ls.setService(tags);
					} else if(tel.text().contains("商家地址")) {
						tel.remove();
						ls.setAddress(el.text().trim());
					} else if(tel.text().contains("联系电话")) {
						tel.remove();
						ls.setPhone(el.text().trim());
					} else if(tel.text().contains("所在区域")) {
						tel.remove();
						String area = el.text().trim();
						if(area.contains(">")) {
							ls.setProvinceName(area.split(">")[0]);
							ls.setCityName(area.split(">")[1]);
						} else {
							ls.setProvinceName(area);
						}
					}
				}
			}
		}
		

		
		els = doc.select("#tconn_1 ul.lis_un li a img");
		if(null != els && !els.isEmpty()) {
			String imgs = "";
			for(Element el : els) {
				if(imgs.length() > 0) imgs += ";";
				imgs+=el.attr("src");
			}
			ls.setImgs(imgs);
		}
		
		Element el = doc.getElementById("tconn_1");
		if(null != el) {
			els = el.getElementsByTag("script");
			if(null != els ) els.remove();
			Element tel  = el.getElementById("BAIDU_SSP__wrapper_u2834905_0");
			if(null != tel) tel.remove();
			els = el.select("ul.lis_un");
			if(null != els ) els.remove();
			ls.setDescription(HtmlUtil.delNoteTag(el.html()));
		}
		return ls;
	}
	
	
	/**
	private void getNjlArea(String url,String type) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocumentByIp(url);
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("1");
			ge.setDescription("获取Score 错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		Map<String, String> map = new HashMap<String, String>();
		Elements els = doc.select(".list-group li ul[class=list-inline] li a");
		boolean flag = true;
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				if(el.text().contains("所有区域")) {
					flag = true;
					continue;
				} else if(el.text().contains("所有主题")) {
					flag = false;
					break;
				}
				if(flag) {
					map.put(el.attr("href"), el.text());
				}
			}
		}
		
		for(String durl : map.keySet()) {
			getNjlPage(durl, map.get(durl));
		}
	}
	
	private void getNjlPage(String url,String city) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocumentByIp(url);
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("2");
			ge.setDescription("获取Score 错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		Map<String, String> map = new HashMap<String, String>();
		Elements els = doc.select(".list-group li ul[class=list-inline] li a");
		boolean flag = true;
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				if(el.text().contains("所有区域")) {
					flag = true;
					continue;
				} else if(el.text().contains("所有主题")) {
					flag = false;
					break;
				}
				if(flag) {
					map.put(el.attr("href"), el.text());
				}
			}
		}
		
		for(String durl : map.keySet()) {
			
		}
	}
*/
	private void getNjlPage(String url, int page) {
		//#page#
		for(int i =0; i< page; i++) {
			getNjlList(url.replace("#page#", i+""));
		}
	}
	private void getNjlList(String url) {
		Document doc = null;
		try {
			doc = GenericGrap.getPostDocByIp(url);
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("1");
			ge.setDescription("获取List 错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		List<LifeShop> lns = new ArrayList<LifeShop>();
		Elements tels = doc.select("li a");
		if(null != tels && !tels.isEmpty()) {
			for(Element el : tels) {
				LifeShop ls = new LifeShop();
				ls.setSource(source);
				ls.setSourceUrl(baseUrl + el.attr("href"));
				// 获取封面
				Elements els = el.select("img.media-object");
				if(null != els && !els.isEmpty()) {
					ls.setCover(els.first().attr("src"));
				}
				els = el.select("div.media-body h4");
				if(null != els && !els.isEmpty()) {
					ls.setName(els.first().text());
				}
				els = el.select("div.media-body p.margin-top-10");
				if(null != els && !els.isEmpty()) {
					ls.setSummary(els.first().text());
				}
				els = el.select("div.media-body div span.pull-left");
				if(null != els && !els.isEmpty()) {
					Elements spanels = els.select("span");
					if(spanels !=null) spanels.remove();
					ls.setAddress(els.text().trim());
				}
				lns.add(ls);
			}
		}
		
		for(LifeShop li : lns) {
			try {
				getNjlDetail(li);
			} catch (Exception e) {
				lns.remove(li);
				e.printStackTrace();
			}
		}
		
		if(lns.size() > 0) insert2ShopDb(lns);
	}

	private LifeShop getNjlDetail(LifeShop ls) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocumentByIp(ls.getSourceUrl());
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(ls.getSourceUrl());
			ge.setType("5");
			ge.setContent(JSON.toJSONString(ls));
			ge.setDescription("获取Detail 错误："+e.getMessage());
			insert2ErrDb(ge);
			return null;
		}
		Elements els = doc.select("div.carousel-inner img");
		if(null != els && !els.isEmpty()) {
			String imgs = "";
			for(Element el : els) {
				if(imgs.length() > 0) imgs += ";";
				imgs+=el.attr("src");
			}
			ls.setImgs(imgs);
		}
		
		els = doc.select("#home div");
		if(null != els && !els.isEmpty()) {
			ls.setDescription(els.first().html());
		}
		return ls;
	}
	private void getCaizaiyuanPage(String url,String type, int page) {
		for(int i = 1; i< page+1; i++) {
			getCaizaiyuanList(url + "&PID=" + i, type);
		}
	}
	
	private void getCaizaiyuanList(String url, String type) {
		Document doc = null;
		try {
			doc = GenericGrap.getDoc(url, "gb2312");
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("1");
			ge.setDescription("获取Score 错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		List<LifeShop> lns = new ArrayList<LifeShop>();
		Elements els = doc.select("table table table table .tu p a");
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				LifeShop ls = new LifeShop();
				ls.setSource(source);
				ls.setSourceUrl(baseUrl+ el.attr("href"));
				String title = el.attr("title");
				ls.setName(title.substring(title.indexOf("]")+1));
				ls.setProvinceName("重庆");
				if(title.indexOf("]") > -1) {
					ls.setCityName(title.substring(0, title.indexOf("]")).replace("[", ""));
				} else {
					System.out.println(title + "=" + ls.getSourceUrl());
				}
				// 获取图片
				Elements imgels = el.getElementsByTag("img");
				if(null == imgels || imgels.isEmpty()) {
					continue;
				} else {
					ls.setCover(baseUrl + imgels.first().attr("src"));
				}
				ls.setTypeName(type);
				lns.add(ls);
			}
		}
		
		for(LifeShop li : lns) {
			try {
				getCaizaiyuanDetail(li);
			} catch (Exception e) {
				lns.remove(li);
				e.printStackTrace();
			}
		}
		
		if(lns.size() > 0) insert2ShopDb(lns);
	}
	
	private LifeShop getCaizaiyuanDetail(LifeShop li) {
		Document doc = null;
		try {
			doc = GenericGrap.getDoc(li.getSourceUrl(), "gb2312");
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(li.getSourceUrl());
			ge.setType("1");
			ge.setContent(JSON.toJSONString(li));
			ge.setDescription("获取Score 错误："+e.getMessage());
			insert2ErrDb(ge);
			return null;
		}
		Elements els = doc.select("table table table table .zi p font");
		if(null != els && !els.isEmpty()) {
			String[] strs = els.first().text().split("加入时间：");
			if(strs.length > 1) {
				li.setPublishTime(strs[1]);
			}
			els = doc.select("#table2 table .wen font");
			if(null != els && !els.isEmpty()) {
				String content = els.first().html();
				content = content.replaceAll("uploadfile", baseUrl+"uploadfile");
				content = content.replaceAll("\\.\\./http", "http").replaceAll("/http", "http");
				li.setDescription(content);
			}
		}
		return li;
	}
	
	private void getKuaiBaoPage(String url, int page) {
		for(int i = 1; i< page+1; i++) {
			getKuaiBaoList(url + "?PID=" + i);
		}
	}
	private void getKuaiBaoList(String url) {
		Document doc = null;
		try {
			doc = GenericGrap.getDoc(url, "gb2312");
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("1");
			ge.setDescription("获取Score 错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		List<LifeNews> lns = new ArrayList<LifeNews>();
		Elements els = doc.select("#table2 table table tbody tr .book a");
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				LifeNews ln = new LifeNews();
				ln.setSource(source);
				ln.setSourceUrl(baseUrl+ el.attr("href"));
//				String title = el.attr("title");
//				try {
//					title = new String(title.getBytes(), "utf-8");
//				} catch (UnsupportedEncodingException e) {
//					// TODO Auto-generated catch block
//					e.printStackTrace();
//				}
				ln.setTitle(el.attr("title"));
				lns.add(ln);
			}
		}
		
		for(LifeNews li : lns) {
			try {
				getKuaiBaoDetail(li);
			} catch (Exception e) {
				lns.remove(li);
				e.printStackTrace();
			}
		}
		
		if(lns.size() > 0) insert2NewsDb(lns);
	}

	private LifeNews getKuaiBaoDetail(LifeNews li) {
		Document doc = null;
		try {
			doc = GenericGrap.getDoc(li.getSourceUrl(), "gb2312");
//			doc = GenericGrap.getDocumentByIp(li.getSourceUrl());
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(li.getSourceUrl());
			ge.setType("1");
			ge.setContent(JSON.toJSONString(li));
			ge.setDescription("获取Score 错误："+e.getMessage());
			insert2ErrDb(ge);
			return null;
		}
		Elements els = doc.select("table table table table .zi p font");
		if(null != els && !els.isEmpty()) {
			String[] strs = els.first().text().split("加入时间：");
			if(strs.length > 1) {
				try {
					li.setPublishTime(DateUtil.string2Date(strs[1], "yyyy-MM-dd HH:mm:ss"));
				} catch (ParseException e) {
					e.printStackTrace();
				}
			}
			els = doc.select("#table2 table .wen font");
			if(null != els && !els.isEmpty()) {
				String content = els.first().html();
				content = content.replaceAll("uploadfile", baseUrl+"uploadfile");
				content = content.replaceAll("\\.\\./http", "http").replaceAll("/http", "http");
				li.setContent(content);
			}
			// 判断是否有图片，有则获取第一张图片作为封面
			els = els.first().getElementsByTag("img");
			if(null != els && !els.isEmpty()) {
				String content = els.first().attr("src");
				content = content.replaceAll("uploadfile", baseUrl+"uploadfile");
				content = content.replaceAll("\\.\\./http", "http").replaceAll("/http", "http");
				li.setCover(content);
				// 获取宽度高度
				Map<Integer, Integer> map = ImageUtil.getImgWidth(li.getCover());
				if(null != map) {
					for(int key : map.keySet()) {
						li.setCoverWidth(key);
						li.setCoverHeight(map.get(key));
					}
				}
			}
		}
		return li;
	}
	

	public void insert2ShopDb(List<LifeShop> lses) {
		String sql = "insert into life_shop(source_url,category_name,type_name,flavor,address,category_ids,shop_hours,name,source,star,cover,traffic,province_name,spend_perhead,service,environment,phone,description,publish_time,city_name,summary,imgs) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
		PreparedStatement ps = null;
		try {
			ps = getConn().prepareStatement(sql);
			for(LifeShop cc : lses) {
				ps.setString(1, cc.getSourceUrl());
				ps.setString(2, cc.getCategoryName());
				ps.setString(3, cc.getTypeName());
				ps.setString(4, cc.getFlavor());
				ps.setString(5, cc.getAddress());
				ps.setString(6, cc.getCategoryIds());
				ps.setString(7, cc.getShopHours());
				ps.setString(8, cc.getName());
				ps.setString(9, cc.getSource());
				ps.setString(10, cc.getStar());
				ps.setString(11, cc.getCover());
				ps.setString(12, cc.getTraffic());
				ps.setString(13, cc.getProvinceName());
				ps.setString(14, cc.getSpendPerhead());
				ps.setString(15, cc.getService());
				ps.setString(16, cc.getEnvironment());
				ps.setString(17, cc.getPhone());
				ps.setString(18, cc.getDescription());
				ps.setString(19, cc.getPublishTime());
				ps.setString(20, cc.getCityName());
				ps.setString(21, cc.getSummary());
				ps.setString(22, cc.getImgs());
				ps.addBatch();
			}
			ps.executeBatch();
			System.out.print("，插入数据库"+lses.size());
		} catch (Exception e) {
			e.printStackTrace();
			GrapErr ge = new GrapErr();
			ge.setType("5");
			ge.setContent(JSON.toJSONString(lses));
			ge.setDescription("insert DB 错误：" + e.getMessage());
			insert2ErrDb(ge);
			return;
		} finally {
			if (null != ps) {
				try {
					ps.close();
				} catch (SQLException e) {
				}
			}
		}
	}
	public void insert2NewsDb(List<LifeNews> nis) {
		String sql = "insert into life_news_tour(author,title,summary,content,publish_time,source,source_url,type_name,create_time,cover_height,cover_width,cover) values(?,?,?,?,?,?,?,?,?,?,?,?)";
		PreparedStatement ps = null;
		try {
			ps = getConn().prepareStatement(sql);
			List<String> ids = new ArrayList<String>();
			List<String> idxNis = new ArrayList<String>();
			for(LifeNews ni : nis) {
				ps.setString(1, ni.getAuthor());
				ps.setString(2, ni.getTitle());
				ps.setString(3, ni.getSummary());
				ps.setString(4, ni.getContent());
				ps.setString(5, null == ni.getPublishTime() ? DateUtil.date2String(new Date(), "yyyy-MM-dd HH:mm:ss") : DateUtil.date2String(ni.getPublishTime(), "yyyy-MM-dd HH:mm:ss"));
				
				ps.setString(6, ni.getSource());
				ps.setString(7, ni.getSourceUrl());
				ps.setString(8, ni.getTypeName());
				ps.setString(9, DateUtil.date2String(new Date(), "yyyy-MM-dd HH:mm:ss"));
				ps.setInt(10, ni.getCoverHeight());
				ps.setInt(11, ni.getCoverWidth());
				ps.setString(12, ni.getCover());
//				ps.setInt(13, ni.getId());
//				ps.setInt(14, ni.getTypeId());
				ps.addBatch();
				ids.add(ni.getId() + "");
				idxNis.add(JSON.toJSONString(ni, SerializerFeature.WriteDateUseDateFormat).replace("\n", ""));
			}
			ps.executeBatch();
			System.out.print("，插入数据库"+ids.size());
		} catch (Exception e) {
			e.printStackTrace();
			GrapErr ge = new GrapErr();
			ge.setType("5");
			ge.setContent(JSON.toJSONString(nis));
			ge.setDescription("insert DB 错误：" + e.getMessage());
			insert2ErrDb(ge);
			return;
		} finally {
			if (null != ps) {
				try {
					ps.close();
				} catch (SQLException e) {
				}
			}
		}
	}

	
	private Connection getConn() {
		if(null == conn) conn = DbUtil.getNewsConn();
		return conn;
	}
	
	public void insert2ErrDb(GrapErr ge) {
		PreparedStatement grapPs = null;
		try {
			grapPs = DbUtil.getComConn().prepareStatement("insert into grap_err(description,url,content,type) values(?,?,?,?)");
			grapPs.setString(1, ge.getDescription());
			grapPs.setString(2, ge.getUrl());
			grapPs.setString(3, ge.getContent());
			grapPs.setString(4, ge.getType());
			grapPs.execute();
			grapPs.close();
		} catch(Exception e) {
			e.printStackTrace();
		} finally {
			if(null != grapPs) {
				try {
					grapPs.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}
	
	public static List<GrapErr> getErrs() {
		List<GrapErr> univs = new ArrayList<GrapErr>();
		try {
			Connection connection = DbUtil.getComConn();
			Statement statement = connection.createStatement();
			String sql = "select * from grap_err";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				GrapErr univ = new GrapErr();
				univ.setId(rs.getInt("id"));
				univ.setUrl(rs.getString("url"));
				univ.setType(rs.getString("type"));
				univ.setContent(rs.getString("content"));
				univs.add(univ);
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return univs;
	}
	
}
